EMR
Add-step Example
aws emr add-steps \
--cluster-id $CLUSTER_ID \
--steps '[{
"Name": "Example PySpark Job",
"Type": "Spark",
"ActionOnFailure": "CONTINUE",
"Args": [
"--deploy-mode", "cluster",
"--master", "yarn",
"--jars", "s3://path/to/definity-spark-agent-X-X.jar",
"--conf", "spark.plugins=ai.definity.spark.plugin.DefinitySparkPlugin",
"--conf", "spark.definity.server=https://app.definity.run",
"--conf", "spark.definity.api.token='$DEFINITY_API_TOKEN'",
"--conf", "spark.definity.env.name=demo",
"--conf", "spark.definity.pipeline.name=example_pipeline",
"--conf", "spark.definity.pipeline.pit=2024-12-10",
"--conf", "spark.definity.task.name=example_task",
"--conf", "spark.executor.memory=3g",
"--conf", "spark.executor.cores=3",
"s3://path/to/task.py"
]
}]'
Compatibility matrix
| EMR Release | Spark Version | Scala Version | Definity Agent |
|---|---|---|---|
| 7.10.0 | 3.5.5 | 2.12.18 | 3.5_2.12-latest |
| 7.9.0 | 3.5.5 | 2.12.18 | 3.5_2.12-latest |
| 7.8.0 | 3.5.4 | 2.12.18 | 3.5_2.12-latest |
| 7.7.0 | 3.5.3 | 2.12.18 | 3.5_2.12-latest |
| 7.6.0 | 3.5.3 | 2.12.18 | 3.5_2.12-latest |
| 7.5.0 | 3.5.2 | 2.12.18 | 3.5_2.12-latest |
| 7.4.0 | 3.5.1 | 2.12.18 | 3.5_2.12-latest |
| 7.3.0 | 3.5.1 | 2.12.18 | 3.5_2.12-latest |
| 7.2.0 | 3.5.0 | 2.12.17 | 3.5_2.12-latest |
| 7.1.0 | 3.5.0 | 2.12.17 | 3.5_2.12-latest |
| 7.0.0 | 3.5.3 | 2.12.17 | 3.5_2.12-latest |
| 6.15.0 | 3.4.1 | 2.12.15 | 3.4_2.12-latest |
| 6.14.0 | 3.4.1 | 2.12.15 | 3.4_2.12-latest |
| 6.13.0 | 3.4.1 | 2.12.15 | 3.4_2.12-latest |
| 6.12.0 | 3.4.0 | 2.12.15 | 3.4_2.12-latest |
| 6.11.1 | 3.3.2 | 2.12.15 | 3.3_2.12-latest |
| 6.11.0 | 3.3.2 | 2.12.15 | 3.3_2.12-latest |
| 6.11.1 | 3.3.2 | 2.12.15 | 3.3_2.12-latest |
| 6.10.1 | 3.3.1 | 2.12.15 | 3.3_2.12-latest |
| 6.10.0 | 3.3.1 | 2.12.15 | 3.3_2.12-latest |
| 6.9.1 | 3.3.0 | 2.12.15 | 3.3_2.12-latest |
| 6.9.0 | 3.3.0 | 2.12.15 | 3.3_2.12-latest |
| 6.8.1 | 3.3.0 | 2.12.15 | 3.3_2.12-latest |
| 6.8.0 | 3.3.0 | 2.12.15 | 3.3_2.12-latest |
| 6.7.0 | 3.2.1 | 2.12.15 | 3.2_2.12-latest |
| 6.6.0 | 3.2.0 | 2.12.10 | 3.2_2.12-latest |
| 6.5.0 | 3.1.2 | 2.12.10 | 3.1_2.12-latest |
| 6.4.0 | 3.1.2 | 2.12.10 | 3.1_2.12-latest |
| 6.3.1 | 3.1.1 | 2.12.10 | 3.1_2.12-latest |
| 6.3.0 | 3.1.1 | 2.12.10 | 3.1_2.12-latest |
| 6.0.1 | 2.4.4 | 2.12.10 | 2.4_2.12-latest |
| 6.0.0 | 2.4.4 | 2.11.12 | 2.4_2.11-latest |